'''
Created on Apr 19, 2013

@author: LarsoMi1
'''

import itertools
from scipy import stats
import numpy as np

from data_types.InternetUsageSurvey import education_levels, income_levels

_LINE_WIDTH = 80

def print_associations(session, min_support=0.3, min_confidence=0.5):
    people = [p for h in session.households.values() for p in h]
    
    ms_thresh = min_support * len(people)
    #mc_thresh = min_confidence * len(people)
    
    totals_k1 = {}
    for person in people:
        for kv_pair in list(people[0].get_attributes().iterkeys()):
            if person.get_attributes()[kv_pair] < 0:
                continue
            key_kv1 = '_'.join([kv_pair, str(person.get_attributes()[kv_pair])])
            totals_k1[key_kv1] = totals_k1.get(key_kv1, 0) + 1
    
    keys_kv1 = list(totals_k1.iterkeys())
    for key_kv1 in keys_kv1:
        hit_count = totals_k1[key_kv1]
        if hit_count < ms_thresh:
            del totals_k1[key_kv1]
    
    pairs_k2 = list(itertools.combinations(list(totals_k1.iterkeys()), 2))
    totals_k2 = {}
    for person in people:
        for kv_pairs in pairs_k2:
            attr1, exp_val1 = kv_pairs[0].split('_')
            value1 = person.get_attributes()[attr1]
            if int(exp_val1) != value1:
                continue
            attr2, exp_val2 = kv_pairs[1].split('_')
            value2 = person.get_attributes()[attr2]
            if int(exp_val2) != value2:
                continue
            key_kv2 = '&'.join(kv_pairs)
            totals_k2[key_kv2] = totals_k2.get(key_kv2, 0) + 1
    
    keys_kv2 = list(totals_k2.iterkeys())
    for key_kv2 in keys_kv2:
        hit_count = totals_k2[key_kv2]
        if hit_count < ms_thresh:
            del totals_k2[key_kv2]
    
    #aborting this line of investigation: all patterns are wholly uninteresting

def print_statistics(session):
    people = [p for h in session.households.values() for p in h]
    
    print '=' * _LINE_WIDTH
    print 'records (total):    %8d' % session.processed
    print 'records (accepted): %8d' % session.accepted
    print 'records (rejected): %8d' % session.rejected
    print '-' * _LINE_WIDTH
    print 'households:         %8d' % len(session.households)
    print 'people:             %8d' % sum([len(h) for h in session.households.values()])
    print 'people:             %8d' % len(people)
    print '-' * _LINE_WIDTH
    print 'average household size: %1.2f' % (float(session.accepted) / len(session.households))
    print '=' * _LINE_WIDTH
    #print_internet_statistics(session.households.values(), people)
    #print '-' * _LINE_WIDTH
    #print_luddites(session.households.values())
    print '-' * _LINE_WIDTH
    print_work_related_internet_usage(people)
    print '-' * _LINE_WIDTH
    print_gamer_statistics(people)
    print '-' * _LINE_WIDTH
    
    avg = lambda lst: sum(lst) / float(len(lst))
    print 'average age of users (for internet access):'
    print '  total:   %2.2f years' % avg([user.age for user in people])
    print '  users:   %2.2f years' % avg([user.age for user in people if user.internet_access == 'Y'])
    print '  -------- -----------'
    print '  @home:   %2.2f years' % avg([user.age for user in people if user.home_internet_access == 'Y'])
    print '  @work:   %2.2f years' % avg([user.age for user in people if user.work_internet_access == 'Y'])
    print '  @school: %2.2f years' % avg([user.age for user in people if user.school_internet_access == 'Y'])
    print '  -------- -----------'
    print '  desktop: %2.2f years' % avg([user.age for user in people if user.desktop == 'Y'])
    print '  laptop:  %2.2f years' % avg([user.age for user in people if user.laptop == 'Y'])
    print '  tablet:  %2.2f years' % avg([user.age for user in people if user.tablet == 'Y'])
    print '  mobile:  %2.2f years' % avg([user.age for user in people if user.mobile == 'Y'])
    print '  console: %2.2f years' % avg([user.age for user in people if user.console == 'Y'])
    print '  -------- -----------'
    print '  learn:   %2.2f years' % avg([user.age for user in people if user.educational_internet_usage == 'Y'])
    print '  job:     %2.2f years' % avg([user.age for user in people if user.job_search_on_internet == 'Y'])
    print '  health:  %2.2f years' % avg([user.age for user in people if user.research_health_information == 'Y'])
    print '  -------- -----------'
    print '-' * _LINE_WIDTH
    print 'average age of people (medium for news and other information):'
    print '  television:   %2.2f years' % avg([user.age for user in people if user.news_television == 'Y'])
    print '  radio:        %2.2f years' % avg([user.age for user in people if user.news_radio == 'Y'])
    print '  internet:     %2.2f years' % avg([user.age for user in people if user.news_internet == 'Y'])
    print '  print:        %2.2f years' % avg([user.age for user in people if user.news_print == 'Y'])
    print '  conversation: %2.2f years' % avg([user.age for user in people if user.news_conversation == 'Y'])
    print '  other:        %2.2f years' % avg([user.age for user in people if user.news_other == 'Y'])
    print '-' * _LINE_WIDTH
    grade_9 = [user.age for user in people if user.education == '9TH GRADE']
    grade_10 = [user.age for user in people if user.education == '10TH GRADE']
    grade_11 = [user.age for user in people if user.education == '11TH GRADE']
    grade_12 = [user.age for user in people if user.education == '12TH GRADE NO DIPLOMA']
    grads = [user.age for user in people if user.education == 'HIGH SCHOOL GRAD-DIPLOMA OR EQUIV (GED)']
    print '   9th graders:   %2.2f years (%d people)' % (avg(grade_9), len(grade_9))
    print '  10th graders:   %2.2f years (%d people)' % (avg(grade_10), len(grade_10))
    print '  11th graders:   %2.2f years (%d people)' % (avg(grade_11), len(grade_11))
    print '  12th graders:   %2.2f years (%d people)' % (avg(grade_12), len(grade_12))
    print '  H.S. graduates: %2.2f years (%d people)' % (avg(grads), len(grads))
    print '-' * _LINE_WIDTH
    grade_9 = [user.HEFAMINC for user in people if user.education == '9TH GRADE' and 0 < user.HEFAMINC]
    grade_10 = [user.HEFAMINC for user in people if user.education == '10TH GRADE' and 0 < user.HEFAMINC]
    grade_11 = [user.HEFAMINC for user in people if user.education == '11TH GRADE' and 0 < user.HEFAMINC]
    grade_12 = [user.HEFAMINC for user in people if user.education == '12TH GRADE NO DIPLOMA' and 0 < user.HEFAMINC]
    grads = [user.HEFAMINC for user in people if user.education == 'HIGH SCHOOL GRAD-DIPLOMA OR EQUIV (GED)' and 0 < user.HEFAMINC]
    
    print '   9th graders:   %2.2f (%s)' % (avg(grade_9),  income_levels[round(avg(grade_9))])
    print '  10th graders:   %2.2f (%s)' % (avg(grade_10), income_levels[round(avg(grade_10))])
    print '  11th graders:   %2.2f (%s)' % (avg(grade_11), income_levels[round(avg(grade_11))])
    print '  12th graders:   %2.2f (%s)' % (avg(grade_12), income_levels[round(avg(grade_12))])
    print '  H.S. graduates: %2.2f (%s)' % (avg(grads),    income_levels[round(avg(grads))])
    print '=' * _LINE_WIDTH

def print_luddites(households):
    count = 0
    for household in households:
        if household.home_internet_access != 'Y':
            continue
        for person in household:
            if person.home_internet_access == 'N':
                count += 1
    print 'number of people with access to the internet but don\'t use it:', count

def print_internet_statistics(households, people):
    t1 = len([h for h in households if h.home_internet_access == 'Y'])
    #t2 = len([h for h in session.households.values() if h.home_internet_access_2 == 'Y'])
    print 'households with Internet access: %d' % (t1)
    
    print 'internet usage by educational achievement:'
    people_by_education_level = {}
    for level in education_levels:
        people_by_education_level[level] = [p for p in people if p.education == level]
        if people_by_education_level[level]:
            print '  %s (%d)' % (level, len(people_by_education_level[level]))
            haves = [p for p in people_by_education_level[level] if p.home_internet_access == 'Y']
            have_nots = [p for p in people_by_education_level[level] if p.home_internet_access == 'N']
            unknown = [p for p in people_by_education_level[level] if p.home_internet_access not in ('Y', 'N')]
            print '    Y: %d' % len(haves)
            print '    N: %d' % len(have_nots)
            print '    ?: %d' % len(unknown)

#def _get_quads(people, attribute1, responses1, attribute2, responses2):
#    results11 = 0
#    results21 = 0
#    results12 = 0
#    results22 = 0
#    total = 0
#    for person in people:
#        response1 = getattr(person, attribute1)
#        response2 = getattr(person, attribute2)
#        if response1 not in responses1 or response2 not in responses2:
#            continue
#        if response1 == responses1[0]:
#            if response2 == responses2[0]:
#                results11 += 1
#            else:
#                results21 += 1
#        elif response2 == responses2[0]:
#            results12 += 1
#        else:
#            results22 += 1
#        total += 1
#    
#    results1x = results11 + results12
#    resultsx1 = results11 + results21
#    results2x = total - results1x
#    resultsx2 = total - resultsx1
#    
#    return total, resultsx1, resultsx2, results1x, results2x, results11, results12, results21, results22


def print_gamer_statistics(people):
#    total, males, females, gamers, lamers = _get_quads(people, 'sex', ['M', 'F'], 'console', ['Y', 'N'])
    male_console = 0
    male_no_console = 0
    female_console = 0
    female_no_console = 0
    total = 0
    for person in people:
        if person.sex not in ['M', 'F'] or person.console not in ['Y', 'N']:
            continue
        if person.sex == 'M':
            if person.console == 'Y':
                male_console += 1
            else:
                male_no_console += 1
        elif person.console == 'Y':
            female_console += 1
        else:
            female_no_console += 1
        total += 1
    
    gamers = male_console + female_console
    males = male_console + male_no_console
    lamers = total - gamers
    females = total - males
    
    e = [((males * gamers) / float(total)), ((females * gamers) / float(total)),
         ((males * lamers) / float(total)), ((females * lamers) / float(total))]
    o = [male_console, female_console, male_no_console, female_no_console]
    
    print """internet access through game consoles by gender: contingency table
            males          females
    gamers [%5d (%5d), %5d (%5d)]
    lamers [%5d (%5d), %5d (%5d)]""" % tuple(int(item) for sublist in zip(o, e) for item in sublist)
    
    chi2 = stats.chisquare(np.array(o), np.array(e))[0]
    print 'X^2: %1.2f (%s)' % (chi2, 'correlated' if 10.828 < chi2 else 'independent') #10.828 for 1 degree of freedom

def print_work_related_internet_usage(people):
    both = 0
    neither = 0
    access_only = 0
    usage_only = 0
    total = 0
    for person in people:
        if person.work_internet_access == 'BLANK' or person.job_search_on_internet == 'BLANK':
            continue
        if person.work_internet_access == 'Y':
            if person.job_search_on_internet == 'Y':
                both += 1
            else:
                access_only += 1
        elif person.job_search_on_internet == 'Y':
            usage_only += 1
        else:
            neither += 1
        total += 1
    
    usage = both + usage_only
    access = both + access_only
    no_usage = total - usage
    no_access = total - access
    
    e = [((access * usage) / float(total)), ((no_access * usage) / float(total)),
         ((access * no_usage) / float(total)), ((no_access * no_usage) / float(total))]
    o = [both, usage_only, access_only, neither]
    
    print """internet access at work and job searching: contingency table
            Access         !Access
     Usage [%5d (%5d), %5d (%5d)]
    !Usage [%5d (%5d), %5d (%5d)]""" % tuple(int(item) for sublist in zip(o, e) for item in sublist)
    
    chi2 = stats.chisquare(np.array(o), np.array(e))[0]
    print 'X^2: %1.2f (%s)' % (chi2, 'correlated' if 10.828 < chi2 else 'independent') #10.828 for 1 degree of freedom

def print_session(session, **kwargs):
    print '=' * _LINE_WIDTH
    for household in session.households.values():
        if household.id in [kwargs.get('hid', None)] + kwargs.get('hids', []):
            print_household(household)
            print '=' * _LINE_WIDTH

def print_household(household):
    print household
    print '-' * _LINE_WIDTH
    for person in household:
        print '  %s' % str(person)

def print_sanity_error(error, household=None, verbose=0):
    if 1 < verbose and household:
        print '=' * _LINE_WIDTH
    if verbose:
        print 'removing household (%s)' % str(error)
    if 1 < verbose and household:
        print '-' * _LINE_WIDTH
        print_household(household)

def print_sanity_summary(num_pruned):
    print '=' * _LINE_WIDTH
    print 'pruned %d households (troublesome data)' % num_pruned
    print '=' * _LINE_WIDTH
